{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "02154.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/yananma/5_programs_per_day/blob/master/02154.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n1nOip-QvslR",
        "colab_type": "text"
      },
      "source": [
        "## 6.3 语言模型数据集 ( 周杰伦专辑歌词 )"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p6f4-IB7wAhK",
        "colab_type": "text"
      },
      "source": [
        "### 6.3.1 读取数据集"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "snposp3EHGf8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch \n",
        "import random \n",
        "import zipfile"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "se345MfAHxIk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!mkdir ../../data"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ycfw0j3zH3UC",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 101
        },
        "outputId": "ed51a7b4-a22f-478b-d643-e5bcc4f558cf"
      },
      "source": [
        "!git clone https://github.com/ShusenTang/Dive-into-DL-PyTorch.git"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'Dive-into-DL-PyTorch'...\n",
            "remote: Enumerating objects: 1692, done.\u001b[K\n",
            "Receiving objects:   0% (1/1692)   \rReceiving objects:   1% (17/1692)   \rReceiving objects:   2% (34/1692)   \rReceiving objects:   3% (51/1692)   \rReceiving objects:   4% (68/1692)   \rReceiving objects:   5% (85/1692)   \rReceiving objects:   6% (102/1692)   \rReceiving objects:   7% (119/1692)   \rReceiving objects:   8% (136/1692)   \rReceiving objects:   9% (153/1692)   \rReceiving objects:  10% (170/1692)   \rReceiving objects:  11% (187/1692)   \rReceiving objects:  12% (204/1692)   \rReceiving objects:  13% (220/1692)   \rReceiving objects:  14% (237/1692)   \rReceiving objects:  15% (254/1692), 172.01 KiB | 319.00 KiB/s   \rReceiving objects:  16% (271/1692), 172.01 KiB | 319.00 KiB/s   \rReceiving objects:  17% (288/1692), 172.01 KiB | 319.00 KiB/s   \rReceiving objects:  18% (305/1692), 172.01 KiB | 319.00 KiB/s   \rReceiving objects:  18% (318/1692), 1.20 MiB | 1.12 MiB/s   \rReceiving objects:  19% (322/1692), 1.20 MiB | 1.12 MiB/s   \rReceiving objects:  20% (339/1692), 1.20 MiB | 1.12 MiB/s   \rReceiving objects:  21% (356/1692), 1.20 MiB | 1.12 MiB/s   \rReceiving objects:  22% (373/1692), 1.20 MiB | 1.12 MiB/s   \rReceiving objects:  23% (390/1692), 1.20 MiB | 1.12 MiB/s   \rReceiving objects:  24% (407/1692), 1.20 MiB | 1.12 MiB/s   \rReceiving objects:  25% (423/1692), 1.20 MiB | 1.12 MiB/s   \rReceiving objects:  26% (440/1692), 1.20 MiB | 1.12 MiB/s   \rReceiving objects:  27% (457/1692), 1.20 MiB | 1.12 MiB/s   \rReceiving objects:  28% (474/1692), 1.20 MiB | 1.12 MiB/s   \rReceiving objects:  29% (491/1692), 1.20 MiB | 1.12 MiB/s   \rReceiving objects:  30% (508/1692), 1.20 MiB | 1.12 MiB/s   \rReceiving objects:  31% (525/1692), 1.20 MiB | 1.12 MiB/s   \rReceiving objects:  32% (542/1692), 1.20 MiB | 1.12 MiB/s   \rReceiving objects:  33% (559/1692), 1.20 MiB | 1.12 MiB/s   \rReceiving objects:  34% (576/1692), 1.20 MiB | 1.12 MiB/s   \rReceiving objects:  35% (593/1692), 1.20 MiB | 1.12 MiB/s   \rReceiving objects:  36% (610/1692), 1.20 MiB | 1.12 MiB/s   \rReceiving objects:  37% (627/1692), 1.20 MiB | 1.12 MiB/s   \rReceiving objects:  38% (643/1692), 1.20 MiB | 1.12 MiB/s   \rReceiving objects:  39% (660/1692), 1.20 MiB | 1.12 MiB/s   \rReceiving objects:  40% (677/1692), 1.20 MiB | 1.12 MiB/s   \rReceiving objects:  41% (694/1692), 1.20 MiB | 1.12 MiB/s   \rReceiving objects:  42% (711/1692), 6.64 MiB | 4.15 MiB/s   \rReceiving objects:  43% (728/1692), 6.64 MiB | 4.15 MiB/s   \rReceiving objects:  43% (730/1692), 13.11 MiB | 6.13 MiB/s   \rReceiving objects:  44% (745/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  45% (762/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  46% (779/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  47% (796/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  48% (813/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  49% (830/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  50% (846/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  51% (863/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  52% (880/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  53% (897/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  54% (914/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  55% (931/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  56% (948/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  57% (965/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  58% (982/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  59% (999/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  60% (1016/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  61% (1033/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  62% (1050/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  63% (1066/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  64% (1083/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  65% (1100/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  66% (1117/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  67% (1134/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  68% (1151/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  69% (1168/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  70% (1185/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  71% (1202/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  72% (1219/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  73% (1236/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  74% (1253/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  75% (1269/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  76% (1286/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  77% (1303/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  78% (1320/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  79% (1337/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  80% (1354/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  81% (1371/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  82% (1388/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  83% (1405/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  84% (1422/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  85% (1439/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  86% (1456/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  87% (1473/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  88% (1489/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  89% (1506/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  90% (1523/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  91% (1540/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  92% (1557/1692), 19.53 MiB | 7.29 MiB/s   \rReceiving objects:  93% (1574/1692), 19.53 MiB | 7.29 MiB/s   \rremote: Total 1692 (delta 0), reused 0 (delta 0), pack-reused 1692\u001b[K\n",
            "Receiving objects: 100% (1692/1692), 25.29 MiB | 8.21 MiB/s, done.\n",
            "Resolving deltas: 100% (975/975), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dDecvkUwH_Sp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!cp Dive-into-DL-PyTorch/data/jaychou_lyrics.txt.zip ../../data"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WFjug-2MITHx",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "e62f84c6-b6b0-4d58-e6e4-40489613287e"
      },
      "source": [
        "with zipfile.ZipFile('../../data/jaychou_lyrics.txt.zip') as zin:\n",
        "    with zin.open('jaychou_lyrics.txt') as f:\n",
        "        corpus_chars = f.read().decode('utf-8')\n",
        "corpus_chars[:40]"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'想要有直升机\\n想要和你飞到宇宙去\\n想要和你融化在一起\\n融化在宇宙里\\n我每天每天每'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rWZYPfrKIwRu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "corpus_chars = corpus_chars.replace('\\n', ' ').replace('\\r', ' ')\n",
        "corpus_chars = corpus_chars[0:10000]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hgIAwyESJhgJ",
        "colab_type": "text"
      },
      "source": [
        "### 6.3.2 建立字符索引"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QjtoNBaPJraT",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "2defd5e8-8e08-4f76-d383-c4fbad50d5d5"
      },
      "source": [
        "idx_to_char = list(set(corpus_chars))\n",
        "char_to_idx = dict([(char, i) for i, char in enumerate(idx_to_char)])\n",
        "vocab_size = len(char_to_idx)\n",
        "vocab_size"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1027"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H8dPubypKORY",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        },
        "outputId": "8508054c-4f9d-4143-d5d3-c332d03d00e9"
      },
      "source": [
        "corpus_indices = [char_to_idx[char] for char in corpus_chars]\n",
        "sample = corpus_indices[:20]\n",
        "print('chars:', ''.join([idx_to_char[idx] for idx in sample]))\n",
        "print('indices:', sample)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "chars: 想要有直升机 想要和你飞到宇宙去 想要和\n",
            "indices: [181, 383, 133, 494, 31, 937, 162, 181, 383, 1024, 740, 819, 939, 209, 294, 873, 162, 181, 383, 1024]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zmdtZp7XKm8z",
        "colab_type": "text"
      },
      "source": [
        "### 6.3.3 时序数据的采样"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ycbNi7ecKuYs",
        "colab_type": "text"
      },
      "source": [
        "#### 1 随机采样"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JAHwXTZQK1AR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def data_iter_random(corpus_indices, batch_size, num_steps, device=None):\n",
        "    num_examples = (len(corpus_indices) - 1) // num_steps \n",
        "    epoch_size = num_examples // batch_size \n",
        "    example_indices = list(range(num_examples))\n",
        "    random.shuffle(example_indices)\n",
        "\n",
        "    def _data(pos):\n",
        "        return corpus_indices[pos: pos + num_steps]\n",
        "    if device is None:\n",
        "        device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "\n",
        "    for i in range(epoch_size):\n",
        "        i = i * batch_size \n",
        "        batch_indices = example_indices[i: i + batch_size]\n",
        "        X = [_data(j * num_steps) for j in batch_indices]\n",
        "        Y = [_data(j * num_steps + 1) for j in batch_indices]\n",
        "        yield torch.tensor(X, dtype=torch.float32, device=device), torch.tensor(Y, dtype=torch.float32, device=device)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tHiDjyPq-cg1",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 185
        },
        "outputId": "505ca017-bbbd-4224-9f6d-765359414675"
      },
      "source": [
        "my_seq = list(range(30))\n",
        "for X, Y in data_iter_random(my_seq, batch_size=2, num_steps=6):\n",
        "    print('X: ', X, '\\nY: ', Y, '\\n')"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "X:  tensor([[ 0.,  1.,  2.,  3.,  4.,  5.],\n",
            "        [12., 13., 14., 15., 16., 17.]], device='cuda:0') \n",
            "Y:  tensor([[ 1.,  2.,  3.,  4.,  5.,  6.],\n",
            "        [13., 14., 15., 16., 17., 18.]], device='cuda:0') \n",
            "\n",
            "X:  tensor([[18., 19., 20., 21., 22., 23.],\n",
            "        [ 6.,  7.,  8.,  9., 10., 11.]], device='cuda:0') \n",
            "Y:  tensor([[19., 20., 21., 22., 23., 24.],\n",
            "        [ 7.,  8.,  9., 10., 11., 12.]], device='cuda:0') \n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SsA_wgtQ-7JI",
        "colab_type": "text"
      },
      "source": [
        "#### 2 相邻采样"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dCVNO2_h_agj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def data_iter_consecutive(corpus_indices, batch_size, num_steps, device=None):\n",
        "    if device is None:\n",
        "        device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "    corpus_indices = torch.tensor(corpus_indices, dtype=torch.float32, device=device)\n",
        "    data_len = len(corpus_indices)\n",
        "    batch_len = data_len // batch_size \n",
        "    indices = corpus_indices[0: batch_size*batch_len].view(batch_size, batch_len)\n",
        "    epoch_size = (batch_len - 1) // num_steps \n",
        "    for i in range(epoch_size):\n",
        "        i = i * num_steps \n",
        "        X = indices[:, i: i + num_steps]\n",
        "        Y = indices[:, i + 1: i + num_steps + 1]\n",
        "        yield X, Y"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dP90qodIAl0I",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 185
        },
        "outputId": "5c08192d-d9e5-446c-98ee-2b7c5358314d"
      },
      "source": [
        "for X, Y in data_iter_consecutive(my_seq, batch_size=2, num_steps=6):\n",
        "    print('X: ', X, '\\nY', Y, '\\n')"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "X:  tensor([[ 0.,  1.,  2.,  3.,  4.,  5.],\n",
            "        [15., 16., 17., 18., 19., 20.]], device='cuda:0') \n",
            "Y tensor([[ 1.,  2.,  3.,  4.,  5.,  6.],\n",
            "        [16., 17., 18., 19., 20., 21.]], device='cuda:0') \n",
            "\n",
            "X:  tensor([[ 6.,  7.,  8.,  9., 10., 11.],\n",
            "        [21., 22., 23., 24., 25., 26.]], device='cuda:0') \n",
            "Y tensor([[ 7.,  8.,  9., 10., 11., 12.],\n",
            "        [22., 23., 24., 25., 26., 27.]], device='cuda:0') \n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}